# 起点月票榜TOP100
import datetime
import logging

import requests
import re
import json
import time
import mysql.connector
import os
import configparser

current_path = os.path.dirname(__file__)

config = configparser.ConfigParser()
config.read(current_path + '/config/properties.conf150')
logging.basicConfig(level=logging.DEBUG,
                    format="%(asctime)s %(name)s %(levelname)s %(message)s",
                    datefmt='%Y-%m-%d  %H:%M:%S %a')

mydb = ''
mycursor = ''
try:
    mydb = mysql.connector.connect(
        host=config['mysql']['host'],  # 数据库主机地址
        port=config['mysql']['port'],
        user=config['mysql']['user'],  # 数据库用户名
        database=config['mysql']['database'],  # 数据库名称 没有的话插入操作报错
        passwd=config['mysql']['passwd']  # 数据库密码

    )
    mycursor = mydb.cursor(buffered=True)
except:
    mydb = ''
    print("无法连接数据库")

sql = "INSERT INTO FAST_START_REPTILE.reptile_qidian_fiction( `name`, `author`, `genre`, `intro`) VALUES  ( %s, %s, %s, %s)"
ticket_sql = "BEGIN " \
             "SET @id:=0;SELECT @id:=id FROM reptile_qidian_fiction WHERE NAME= %s ;INSERT INTO `FAST_START_REPTILE`.`reptile_qidian_ticket`(`fiction_id`, `month_count`, `rec_count`, `reward_num`) VALUES (@id, %s, %s, %s);" \
             " END"
ticket_sql1 = "SET @id:=0"
ticket_sql2 = "SELECT @id:=id FROM reptile_qidian_fiction WHERE NAME= %s"
ticket_sql3 = "INSERT INTO `FAST_START_REPTILE`.`reptile_qidian_ticket`(`fiction_id`, `month_count`, `rec_count`, `reward_num`, create_time) VALUES (@id, %s, %s, %s,%s)"
http = 'https:'
# 格式化成2016-03-2011:45:39形式
current_time = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
sql_current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())


def request_qidian(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except requests.RequestException:
        return None


def parse_result(html):
    pattern = re.compile(
        '<div class="book-img-box">.*?>(\d+)<.*?<a href="(.*?)".*?<h4><a href=.*?"qd_C40".*?>(.*?)</a></h4>.*?<p class="author">.*?target="_blank".*?>(.*?)</a><em>.</em><a href=.*?>(.*?)</a>.*?<p class="intro">\r *(.*?)\r.*?</li>',
        re.S)
    items = re.findall(pattern, html)
    # print(type(items))
    for item in items:
        # print(http + item[1])
        book_detail = request_qidian(http + item[1])
        yue_piao_find = '<i id="monthCount">(.*?)</i>.*?<i id="recCount">(.*?)</i>.*?<i class="rewardNum" id="rewardNum">(.*?)</i>'
        monthCount = re.findall(yue_piao_find, book_detail, re.S)
        # print(monthCount)
        if not monthCount:
            monthCount = [0, 0, 0]
        yield {
            'No.': item[0],
            'name': item[2],
            'monthCount': monthCount[0][0],
            'recCount': monthCount[0][1],
            'rewardNum': monthCount[0][2],
            'author': item[3],
            'genre': item[4],
            'intro': item[5]
        }
    # print(type(items))





def write_item_to_file(item):
    # print('开始写入数据 ====> ' + str(item))
    if not os.path.exists(current_path + '/his/'):
        os.makedirs(current_path + '/his/')
    with open(current_path + '/his/' + current_time + '.txt', 'a', encoding='UTF-8') as f:
        f.write(json.dumps(item, ensure_ascii=False) + '\n')
        f.close()


def main(page):
    url = 'https://www.qidian.com/rank/yuepiao?page=' + str(page)
    html = request_qidian(url)
    items = parse_result(html)  # 解析过滤我们想要的信息
    # print(items)
    for item in items:
        if mydb:
            param = (item.get('name'), item.get('author'), item.get('genre'), item.get('intro'))
            try:
                mycursor.execute(sql, param)
                # 很重要,不然数据会回退,自增的id已经改变
                mydb.commit()
            except:
                logging.warning("此本小说已存在")
            param1 = (item.get('name'), item.get('monthCount'), item.get('recCount'), item.get('rewardNum'))
            param2 = (item.get('name'),)
            param3 = (item.get('monthCount'), item.get('recCount'), item.get('rewardNum'), sql_current_time)
            # try:
            # mycursor.execute(ticket_sql, param2)
            mycursor.execute(ticket_sql1, ())
            mycursor.execute(ticket_sql2, param2)
            mycursor.execute(ticket_sql3, param3)
            # 很重要,不然数据会回退,自增的id已经改变
            mydb.commit()
            # except:
            #     print("存储月票数据失败")
        else:
            logging.info("无法连接到数据库,开始将信息写入临时文件")
            write_item_to_file(item)


def load_his_data():
    file_names = os.listdir(current_path + '/his/')
    if not file_names:
        logging.info("无未入库历史文件!")
    for file_name in file_names:
        logging.info("开始读取未入库的历史数据,文件为" + file_name)
        file = open(current_path + '/his/' + file_name, encoding='UTF-8')

        while True:
            text = file.readline()  # 只读取一行内容
            # 判断是否读取到内容
            if not text:
                break
            msg = eval(text)
            file_time = datetime.datetime.strptime(file_name.split(".")[0], "%Y-%m-%d-%H-%M-%S")
            param1 = (msg.get('name'), msg.get('monthCount'), msg.get('recCount'), msg.get('rewardNum'))
            param2 = (msg.get('name'),)
            param3 = (msg.get('monthCount'), msg.get('recCount'), msg.get('rewardNum'), file_time)
            # try:
            # mycursor.execute(ticket_sql, param2)
            mycursor.execute(ticket_sql1, ())
            mycursor.execute(ticket_sql2, param2)
            mycursor.execute(ticket_sql3, param3)
            # 很重要,不然数据会回退,自增的id已经改变
            mydb.commit()
            # print(type(msg))
            # 每读取一行的末尾已经有了一个 `\n`
            # print(text, end=""),
        file.close()
        os.remove("current_path+'/his/'" + file_name)


def start():
    if mydb:
        # 将历史记录导入数据库,并删除删除历史文件
        logging.info("可以连接库,开始解析历史文件!")
        load_his_data()
    for i in range(1, 6):
        main(i)
    logging.info("解析结束!")


if __name__ == "__main__":
    start()
